# coding:utf-8
import os
import re
import json
import requests
import sys


class Kr36(object):
    def __init__(self):
        self.url = 'http://36kr.com/'
        self.pattern = re.compile('<script>var props=(.*?)</script>', re.S)

    def get_page(self, url):
        _path = os.path.join(os.path.abspath('.'), '../')
        sys.path.append(_path)
        _ei = __import__('Utils.C002_extract_info', fromlist=['C002_extract_info'])
        _ua_file = {'pc': '{0}Utils/{1}'.format(_path, _ei.ExtractInfo.ua_file['pc'])}
        _user_agent = _ei.ExtractInfo.get_useragent(file=_ua_file)
        _response = requests.get(url, headers=_user_agent, verify=False)

        return _response.text

    def parse_data(self, str_data):
        _result = self.pattern.findall(str_data)[0]
        with open('ext/E1_36kr_js_1.json', mode='w', encoding='utf-8') as f:
            f.write(_result)
        _temp_data = re.sub(',locationnal={.*', '', _result)
        with open('ext/E1_36kr_js_2.json', mode='w', encoding='utf-8') as f:
            f.write(_temp_data)
        _dict_data = json.loads(_temp_data)['feedPostsLatest|post']
        _data_list = []
        for _news in _dict_data:
            _temp = {}
            _temp['title'] = _news['title']
            _temp['cover'] = _news['cover']
            _data_list.append(_temp)

        return _data_list

    def save_data(self, data_list):
        with open('ext/E1_36kr_js_3.json', mode='w', encoding='utf-8')as f:
            for _data in data_list:
                _str_data = json.dumps(_data, ensure_ascii=False) + ',\n'
                f.write(_str_data)

    def run(self):
        _str_data = self.get_page(self.url)
        _data_list = self.parse_data(_str_data)
        self.save_data(_data_list)


def main():
    kr36 = Kr36()
    kr36.run()


if __name__ == '__main__':
    main()
